Instructions -- 23May2021¶
External Sources I/O¶
Github I/O¶
In [1]:
import os
import pickle
from pathlib import Path
from datetime import datetime
from datetime import date
import random
from sklearn.model_selection import train_test_split
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
# Setup Constants
GIT_NEW_BRANCH_NAME = f'Lab8: {date.today()} at {current_time}'
GIT_DEFAULT_BRANCH_NAME = 'main'
GIT_USER_NAME = 'Shahar Raz'
GIT_USER_EMAIL = 'SRazStudent@gmail.com'
GIT_DEFAULT_MESSAGE = f'{GIT_USER_NAME}, {date.today()} at {current_time}'
GIT_SAVE_DESTINATION = '/content/ComputerVision2021B/Lab Reports/Lab8/'
!git init
def importFilesFromGithub(branch_name = GIT_DEFAULT_BRANCH_NAME):
"""returns the directory in-which we'll save the files"""
# if not already imported
if not Path("/content/ComputerVision2021B").exists():
# add the new remote (with id key)
! git remote add CV2021Github https://github.com/ShahaRaz/ComputerVision2021B.git
# create new local directory
! mkdir ComputerVision2021B/
# pulling project to local dir
! git --work-tree=/content/ComputerVision2021B/ pull CV2021Github $branch_name;
return r'/content/ComputerVision2021B/Assignment/Assignment2'
def saveFilesToGithub(message_of_commit = GIT_DEFAULT_MESSAGE,branch_name = GIT_NEW_BRANCH_NAME, user_name = GIT_USER_NAME, user_email = GIT_USER_EMAIL):
def setLocalID():
# adding my name
!git config --global user.name user_name #"Shahar "
!git config --global user.email user_email #"SRazStudent@gmail.com"
#__ changing use name: __
# !git config --global --unset user.name
# !git config --global user.name "Shahar ModifiedName "
# list configs
# !git config -l
setLocalID()
# create new branch
if branch_name is not GIT_DEFAULT_BRANCH_NAME:
!git checkout -b $branch_name
# add files to commit
!git add /content/ComputerVision2021B/*
# #checking the status
# !git status
# Commit all Changes
!git commit -a -m message_of_commit #"delete meee"
# Push to remote
!git push CV2021Github $branch_name #master
directory = importFilesFromGithub()
Asmnt2:¶
Imports¶
In [2]:
from matplotlib import pyplot as plt
import numpy as np
import cv2
from scipy.signal import convolve, convolve2d
import math
from torch.utils.data import Dataset, DataLoader
import torch
import albumentations as A
from torchsummary import summary
SIZE_IMAGES_TO = 224 # 224 - IMAGENET SIZE # 240 - (what was asked in asgmnt paper)
## Summary tensor size:
NUM_OF_DIM = 3 # RGB
SUMMARY_TENSOR_SIZE = (NUM_OF_DIM, SIZE_IMAGES_TO, SIZE_IMAGES_TO)
1.Preprocessing:¶
1.0 Label Encoding Names-> int¶
In [3]:
lbl_encoding = []
# Instructions:
# check if label is already in --- --- ---
# if 'Shahar' in lbl_encoding:
# insert new label --- --- --- --- --- ---
# lbl_encoding.append("addMe")
# get index of element --- --- --- --- ---
# lbl_encoding.index('getMyIndex')
# get element from index --- --- --- --- ---
# lbl_encoding[3]
1.1 Dataset Class (Paths)¶
we load paths of photos inorder to keep the photos off the RAM when they're not in use.
Otherwise, we can easly fill up the entire ram (when using large datasets)
In [4]:
class LoadDataFromDisk:
@staticmethod
def split_dataset_3way(X, y,validation_size = 0.30 ,test_size = 0.10):
"""Will Split the data to Train, Test and Validation """
# for example: 90% to training [= 0.1]
if test_size > 0:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size)
# from that 90%, 10% will go to [ validation = 0.1 ]
if validation_size > 0 :
X_train, X_validation, y_train, y_validation = train_test_split(X_train, y_train, test_size = validation_size)
# will leave us with:
# 10% - Test
# 81% - Train
# 9% - Validation
return X_train, X_validation, X_test, y_train, y_validation, y_test
def __init__(self, path_to_images_folder):
"""input: path to Photos folder, which contains folders with photos
output: array of tuples (image, person_name)
"""
# For name encoding (1.0)
label_of_name = -1
# self.path_to_images_folder = path_to_images_folder
self.original_imgs = []
# go over folders inside directory
for dirpath, dirnames, filenames in os.walk(path_to_images_folder):
person_name = dirpath.split('/')[-1]
# skip directory itself
if person_name is 'Photos':
continue
# Adding Pictures Of person_name
for i in range(0,len(filenames)):
# Name Encoding:
# if name not encoded already
if not person_name in lbl_encoding:
# encode name
lbl_encoding.append(person_name)
# get name's index(code)
label_of_name = lbl_encoding.index(person_name)
path_to_image = f'{path_to_images_folder}/{person_name}/{filenames[i]}'
self.original_imgs.append((path_to_image, int(label_of_name)))
def load_data(self, perc_validation=0.2, perc_test=0.0):
# Convert to NP.ndarray
self.original_imgs = np.asarray(self.original_imgs)
# split X-y
X, y = self.original_imgs[:, 0], self.original_imgs[:, 1]
# Convert to encoding to uint8 (save space (1byte))
y = y.astype(np.int32) # WORKS WITH uint89
# # keeping 3way split function ____ for future reference ___
# X_train, X_valid, X_test, y_train, y_valid, y_test = self.split_dataset_3way\
# (X, y, validation_size = perc_validation ,test_size = perc_test)
# train_test_split shuffles the data SHUFFLING ALL DATA HERE
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size = perc_validation,shuffle = True)
return (X_train, y_train), (X_valid, y_valid)
1.2 Present the images & split¶
In [5]:
photos_directory = f'{directory}/Photos'
data = LoadDataFromDisk(photos_directory)
(X_train, y_train), (X_valid, y_valid) = data.load_data()
# Sainity check:
print('train(x,y) \t validation(x,y)')
print(len(X_train), len(y_train), '\t\t\t', len(X_valid), len(y_valid))
print(' Printing train data:')
for image, lbl in zip(X_train, y_train):
plt.imshow(cv2.cvtColor(cv2.imread(image), cv2.COLOR_BGR2RGB))
plt.title(lbl_encoding[lbl])
plt.show()
1.3 Dataloader From Disk W/ Augmentations¶
In [6]:
from torch.utils.data import Dataset, DataLoader
import torch
import albumentations as A
class SuperFaceRecognitionDataset(Dataset):
def __init__(self, x_train, y_train, transforms_album = None, transforms_pytorch = None):
super (SuperFaceRecognitionDataset, self).__init__()
self.x_train = x_train
self.y_train = y_train
self.transforms_album = transforms_album
self.transforms_pytorch = transforms_pytorch
def apply_transforms(self, image):
# Albumentations Transform
if self.transforms_album:
image = self.transforms_album(image=image)["image"]
# Pytorch Transforms
if self.transforms_pytorch:
image = self.transforms_pytorch()
return image
def __getitem__(self,idx):
assert isinstance(idx, (int))
if idx < len(self.x_train) + 1: # (+1 is some sort of bug) TODO: Fixme
# print(self.x_train[idx])
img = cv2.imread(self.x_train[idx])
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# img = pil2np(self.x_train[idx])
label = self.y_train[idx]
# convert label to tensor
label = torch.tensor(label)
# apply_transforms(img)
if self.transforms_album:
img = self.transforms_album(image=img)["image"]
# TODO: Figure out how to apply.
# # Pytorch Transforms
# if self.transforms_pytorch:
# img = self.transforms_pytorch(img)
img = (img.astype(np.float) )/ img.max() # normlize values to [0,1]
img = torch.tensor(img) #return as tensor object (for future )
# img = img.float() ## TODO: FIX THIS STUPID BUG
# label = label.long()
return (img, label)
### WHAT IS WRONG??
# # [Channel , H , W] -> Into The Network -> [Batch, Channel, Hight, Width]
# return (img[np.newaxis, :, :], label) ## TODO : CHECK NEWAXIS DESIGN
else:
print(idx)
return -1
def add_pytorch_transforms(self, transforms_pytorch):
self.transforms_pytorch = transforms_pytorch
def __len__(self):
return len(self.x_train)
def get_lists(self):
return self.x_train, self.y_train
def set_lists(self, X, y):
self.x_train = X
self.y_train = y
In [7]:
transforms_albumentations = A.Compose([
A.Blur(blur_limit=3, p = 0.3),
A.GaussNoise(p=0.2),
A.RGBShift(r_shift_limit=25, g_shift_limit=25, b_shift_limit=25 ,p=0.9),
# A.CLAHE(p=0.3),
A.RandomBrightness(p=0.6,),
A.RandomContrast(p=0.3),
A.RandomGamma(p=0.4),
A.HorizontalFlip(p=0.5),
A.Rotate(limit=10, p = 0.9),
A.SmallestMaxSize(SIZE_IMAGES_TO+1),
A.CenterCrop(SIZE_IMAGES_TO,SIZE_IMAGES_TO),
# A.Normalize(),
])
# Create dataset objects
train_face_dataset = SuperFaceRecognitionDataset(X_train, y_train, transforms_albumentations)
valid_face_dataset = SuperFaceRecognitionDataset(X_valid, y_valid, transforms_albumentations) ## TODO: check if apply transformation on validation set..
plt.imshow(train_face_dataset[2][0])
Out[7]:
In [8]:
print(type(transforms_albumentations))
1.4 Display Data W/ Augmentation¶
In [9]:
BATCH_SIZE = 8
train_loader = DataLoader(train_face_dataset, batch_size=BATCH_SIZE, shuffle=True,num_workers=2)
valid_loader = DataLoader(valid_face_dataset, batch_size=BATCH_SIZE, shuffle=True,num_workers=2)
for images, labels in train_loader:
for image, lbl in zip(images, labels):
plt.imshow(image)
plt.title(lbl_encoding[lbl])
print(image.shape)
plt.show()
1.5 Create OpenCV Model to Idenify faces¶
In [10]:
# https://towardsdatascience.com/face-detection-in-2-minutes-using-opencv-python-90f89d7c0f81
import cv2
# get the XML file from cv2's git repo
if not Path("/content/haarcascade_frontalface_default.xml").exists():
!wget https://raw.githubusercontent.com/opencv/opencv/master/data/haarcascades/haarcascade_frontalface_default.xml -L 'haarcascade_frontalface_default.xml'
face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
def find_face(image, isPrinting=False):
image = (image * 255).astype(np.uint8)
faces = face_cascade.detectMultiScale(image, 1.1, 4)
if isPrinting is True:
plt.imshow(image)
plt.show()
for (x, y, w, h) in faces:
cv2.rectangle(image, (x, y), (x+w, y+h), (255, 0, 0), 2)
plt.imshow(image)
plt.show()
return faces
img = iter(train_loader).next()[0][0].numpy()
img = (img * 255).astype(np.uint8)
plt.imshow(img)
print(type(img), img.shape)
plt.show()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
plt.imshow(gray, 'gray')
plt.show()
faces = face_cascade.detectMultiScale(img, 1.1, 4)
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x+w, y+h), (115, 0, 255), 1)
# Display the output
plt.imshow(img)
plt.show()
1.6 Choose Best representing images & explain¶
Filter bad Photos: As we selected the photos we've choosen them so that will be good for such face detection task, so we didn't filter any photo in this stage.
Augmentations that were not good:
a. color invert (complitly messed up the images, we looked like white-walkers)
b. rotation over 10 degrees. it is extreamly uncommon for the eyes to be below the mouth, must of the shots are in range of 10deg of horizontal.
In [11]:
type(train_face_dataset)
Out[11]:
In [12]:
# for this process we'll check if our face detector detects faces in the image.
counter = 0
for index, img in enumerate(train_face_dataset):
# plt.imshow(img[0])
# plt.show()
if len(find_face(img[0].numpy())) is 0:
# face not found in the pic, remove image
X, y = train_face_dataset.get_lists()
X = np.delete(X, index)
y = np.delete(y, index)
train_face_dataset.set_lists(X, y)
counter += 1
print('deleting image at index: ', index)
print('removed ', counter, ' elements from data.')
# NOTE! Run find_face() is defined in 2.1, please run this cell first.
In [13]:
print(f'training lenght = {len(train_face_dataset)}\nvalid lenght = {len(valid_face_dataset)}')
Asmnt3:¶
Helper Functions¶
Save & Load models¶
In [14]:
# Creating new folder for model saving
! mkdir savedModels/
SAVE_MODELS_DIR = '/content/savedModels'
def save_model(model, optimizer, epoch, loss, path):
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
}, path)
def load_model(path, model, optimizer):
checkpoint = torch.load(path)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']
return epoch, loss
Setup GPU / cuda¶
In [15]:
# Check GPU Given
!nvidia-smi
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
Metrics of evaluation¶
In [16]:
!pip install torchmetrics
import torchmetrics
import torch.nn as nn
In [17]:
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
Monitor - TensorBoard¶
In [18]:
from torch.utils.tensorboard import SummaryWriter
experiment = 1
run = 0
writer_m0_VGG16 = SummaryWriter(f'runs/m0_VGG16_{experiment}')
valid_writer_m0_VGG16 = SummaryWriter(f'runs/m0_VGG16_{experiment}_valid')
# Other 5 models will show at pretrain writer
In [19]:
%load_ext tensorboard
%reload_ext tensorboard
In [20]:
%tensorboard --logdir=runs
Loss Class¶
In [21]:
class FL(nn.Module):
def __init__(self, alpha = 0.2, gamma = 2):
super().__init__()
self.alpha = alpha
self.gamma = gamma
self.CEL = nn.CrossEntropyLoss()
def forward(self, inputs, targets):
CE_loss = self.CEL(inputs, targets) #ce = -ln(pt)
pt = torch.exp(-CE_loss) #e^-ln(pt) = pt
F_loss = self.alpha * (1-pt)**self.gamma * CE_loss
return F_loss
VGG Architecure¶
Imports¶
In [22]:
import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
import tqdm
import pprint
Constants & Transforms¶
In [23]:
# BATCH_SIZE = 5 # Defined in 1.4
LEARNING_RATE = 0.01
N_EPOCHS = 30
N_CLASSES = 2 # Shahar / Messi
# VGG_INPUT_IMAGE_240 = SIZE_IMAGES_TO _BLABLABLLAHERE
VGG_INPUT_CHANNELS = 3 # RGB
device = "cuda" if torch.cuda.is_available() else "cpu"
VGG_types = {
"VGG11": [64, "M", 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
"VGG13": [64, 64, "M", 128, 128, "M", 256, 256, "M", 512, 512, "M", 512, 512, "M"],
"VGG16": [64,64,"M",128,128,"M",256,256,256,"M",512,512,512,"M",512,512,512,"M",],
"VGG19": [64,64,"M",128,128,"M",256,256,256,256,"M",512,512,512,512,"M",512,512,512,512,"M",],
}
# transforms_torch = transforms.Compose([
# transforms.RandomResizedCrop(224),
# transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# transforms.Normalize(mean = [ 0.485, 0.456, 0.406 ],
# std = [ 0.229, 0.224, 0.225 ]),
# ])
# # print(type(transforms_torch))
# train_loader.dataset.add_pytorch_transforms(transforms_torch)
VGG NN Class¶
In [24]:
class VGG_net(nn.Module):
def __init__(self, in_channels=3, num_classes=1000):
super(VGG_net, self).__init__()
self.in_channels = in_channels
self.conv_layers = self.create_conv_layers(VGG_types["VGG16"])
self.fcs = nn.Sequential(
nn.Linear(512 * 7 * 7, 4096),
nn.ReLU(),
nn.Dropout(p=0.5),
nn.Linear(4096, 4096),
nn.ReLU(),
nn.Dropout(p=0.5),
nn.Linear(4096, num_classes),
)
def forward(self, x):
# Fix size..?
# if not (x.shape[2] is SIZE_IMAGES_TO) or not (x.shape[3] is SIZE_IMAGES_TO) :
# x = torch.reshape(x,[-1,VGG_INPUT_CHANNELS,VGG_INPUT_IMAGE_240,VGG_INPUT_IMAGE_240])
# x = x.float()
x = self.conv_layers(x)
x = x.reshape(x.shape[0], -1)
x = self.fcs(x)
return x
def create_conv_layers(self, architecture):
layers = []
in_channels = self.in_channels
for x in architecture:
if type(x) == int:
out_channels = x
layers += [
nn.Conv2d(
in_channels=in_channels,
out_channels=out_channels,
kernel_size=(3, 3),
stride=(1, 1),
padding=(1, 1),
),
nn.BatchNorm2d(x),
nn.ReLU(),
]
in_channels = x
elif x == "M":
layers += [nn.MaxPool2d(kernel_size=(2, 2), stride=(2, 2))]
return nn.Sequential(*layers)
In [25]:
# Simple Demo:
demo_model = VGG_net(in_channels=3, num_classes=1000).to(device)
print(demo_model)
N_deleteme = 16 #(Mini batch size)
print('=' * 400)
x = torch.randn(16, 3, SIZE_IMAGES_TO, SIZE_IMAGES_TO).to(device)
x=x.float()
print(type(x))
print(demo_model(x).shape)
Verfy Model - Overfitting¶
overfitting a small batch:
In [26]:
N_OFITTING_EPOCS = 5000
def over_fit_a_batch(imgs, targets, model, optimizer, epoch, criterion,scheduler):
# Set model to train mode
model.train()
# Best model result (for saving purposes)
best_acc = 0
#rounds in a row in-which we didn't improve
non_improved_rounds = 0
# create new Meters
accuracy = AverageMeter()
losses = AverageMeter()
# Fixing Shape
if not (imgs.shape[2] is SIZE_IMAGES_TO) or not (imgs.shape[3] is SIZE_IMAGES_TO) :
imgs = torch.reshape(imgs,[-1,VGG_INPUT_CHANNELS,SIZE_IMAGES_TO,SIZE_IMAGES_TO])
# TODO: Move this .float() part out of this loop...
imgs = imgs.float()
targets = targets.long()
imgs = imgs.to(device)
targets = targets.to(device)
# Predict - Forward
output = model(imgs)
# Calc loss
loss = criterion(output, targets)
# setup optimizer
optimizer.zero_grad()
# Backward
loss.backward()
# Step the optimizer
optimizer.step()
# Choose single output for each image (highest probability from softmax)
pred = output.argmax(dim=1 , keepdim=True)
# Calc accuracy
acc = torchmetrics.functional.accuracy(pred, targets)
# Update losses & Accuracy logs
losses.update(loss.item(), imgs.size(0))
accuracy.update(acc, imgs.size(0))
# Update Optimizer's Scheduler
scheduler.step(loss)
return acc
# Fetch batch
data, targets = next(iter(train_loader))
# Create an instance of the class
validate_VGG16 = VGG_net(in_channels=3, num_classes=N_CLASSES)
# Set calculations to GPU device
validate_VGG16.cuda() # .to(device) ???? TODO : CHECK THE DIFFERENCE...
#_ ALTERNATIVE _
# m1_MobileNetV3 = m1_MobileNetV3.to(device)
# Loss
# criterion = FL() # TODO : CHECK THE DIFFERENCE...
#_ ALTERNATIVE _
criterion = nn.CrossEntropyLoss()
# Optimizer:
optimizer = torch.optim.Adam(validate_VGG16.parameters(), lr=LEARNING_RATE)
# Scheduler - decrease the learning rate when approaching fine-tuning stage.
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
for epoch in range(N_OFITTING_EPOCS):
if epoch % 500 is 0:
print(f'epoch = {epoch}, acc = {over_fit_a_batch(data, targets, validate_VGG16, optimizer, epoch, criterion,scheduler)}')
We see that our network is not able to overfit over the data, I've try to change many parameters:
batch size | Learning Rate | Augmentations ...
nothing works.. I must continue anyway due..
In [27]:
DEBUG = False
TENSORBOARD_EVERY_N_EPOCHS = 1
CLEAN_CACHE_EVERY_N_EPOCHS = 7
def valid(dataloader, model, criterion, optimizer):
# Set model to test mode
model.eval()
acc_loss = 0
correct = 0
total = 0
losses = AverageMeter()
for I, (imgs, targets) in enumerate(dataloader):
# Fixing Shape
if not (imgs.shape[2] is SIZE_IMAGES_TO) or not (imgs.shape[3] is SIZE_IMAGES_TO) :
imgs = torch.reshape(imgs,[-1,VGG_INPUT_CHANNELS,SIZE_IMAGES_TO,SIZE_IMAGES_TO])
# Maybe Use permute???
imgs = imgs.float()
targets = targets.long()
imgs = imgs.to(device)
targets = targets.to(device)
with torch.no_grad():
output = model(imgs)
loss = criterion(output, targets)
losses.update(loss.item(), imgs.size(0))
pred = output.argmax(dim = 1, keepdim = True)
correct += pred.eq(targets.view_as(pred)).sum()
total += imgs.shape[0]
accuracy = (100 * (correct / total )).item()
return losses.avg, accuracy
def train_single_epoch(train_loader, model, optimizer, epoch, criterion, writer,
validation_writer, name, best_acc, non_improved_rounds = 0,
scheduler = None,is_running_valid = True):
"""
returns the best accuracy on validation that it got so far.
"""
# Set model to train mode
model.train()
#rounds in a row in-which we didn't improve
# create new Meters
accuracy = AverageMeter()
losses = AverageMeter()
for i, (imgs, targets) in enumerate(train_loader):
# Fixing Shape
if not (imgs.shape[2] is SIZE_IMAGES_TO) or not (imgs.shape[3] is SIZE_IMAGES_TO) :
imgs = torch.reshape(imgs,[-1,VGG_INPUT_CHANNELS,SIZE_IMAGES_TO,SIZE_IMAGES_TO])
# TODO: Move this .float() part out of this loop...
imgs = imgs.float()
targets = targets.long()
# Pass data to GPU
imgs = imgs.to(device)
targets = targets.to(device)
# Predict - Forward
output = model(imgs)
# Calc loss
loss = criterion(output, targets)
if DEBUG:
print('output',output.detach().cpu().numpy(),
'shape', output.detach().cpu().numpy().shape )
print('targets',targets.detach().cpu().numpy(),
'shape', targets.detach().cpu().numpy().shape )
# setup optimizer
optimizer.zero_grad()
# Backward
loss.backward()
# Step the optimizer
optimizer.step()
# Choose single output for each image (highest probability from softmax)
pred = output.argmax(dim=1 , keepdim=True)
# Calc accuracy
acc = torchmetrics.functional.accuracy(pred, targets)
# Update losses & Accuracy logs
losses.update(loss.item(), imgs.size(0))
accuracy.update(acc, imgs.size(0))
# Clear cached data:
del imgs
del targets
if epoch % CLEAN_CACHE_EVERY_N_EPOCHS is 0:
# imgs.detach()
# targets.detach()
torch.cuda.empty_cache()
# Update Optimizer's Scheduler
if scheduler:
scheduler.step(loss)
# Validate model in current state
valids_loss, valids_acc = valid(valid_loader, model, criterion, optimizer)
# Write Results To File For TensorBoard Monitorization
# if epoch % TENSORBOARD_EVERY_N_EPOCHS is 0:
# # Write train results
writer.add_scalar(' average training loss', losses.avg, epoch)
writer.add_scalar(' average training accuracy', accuracy.avg, epoch)
# write valid results
validation_writer.add_scalar(' average validating loss', valids_loss, epoch)
validation_writer.add_scalar(' average validating accuracy', valids_acc, epoch)
# save best model's state (only if bigger then 0.6)
if epoch > 5:
if best_acc < valids_acc:
print(f'line120: best_acc = {best_acc}, valids_acc = {valids_acc}')
best_acc = valids_acc
%time save_model(model, optimizer, epoch, losses.avg, f'{SAVED_MODEL_PATH}{name}.wow')
non_improved_rounds = 0
else:
non_improved_rounds += 1
return best_acc, non_improved_rounds
# elif non_improved_rounds is 3:
# load_model()
Train_N_epochs¶
In [28]:
# Set minimum accuracy for saving the model:
MINIMUM_ACC_4RECORDING = 60.0
MAX_ROUNDS_NON_IMPROVED = 5
def train_n_epochs(model, n_epochs, train_loader, valid_loader, criterion, optimizer, scheduler, model_name,
train_writer_tBoard, valid_writer_tBoard,best_acc):
path_to_model = f'{SAVED_MODEL_PATH}{model_name}.wow'
non_improved_rounds = 0
for i in tqdm.tqdm(range(n_epochs), total=n_epochs): # Do total N_EPOCHS
best_acc, non_improved_rounds = train_single_epoch(train_loader, model, optimizer, i, criterion, train_writer_tBoard,
valid_writer_tBoard, model_name, best_acc, non_improved_rounds, scheduler)
# If saved model once atleast
if Path(path_to_model).exists():
# if not improving for 3 epochs, return to best model (lost the way)
if non_improved_rounds is MAX_ROUNDS_NON_IMPROVED:
print(f'line19 epoch #{i}, reverting back to model w/ {best_acc} accuracy')
epoch, loss = load_model(path_to_model, model, optimizer)
non_improved_rounds = 0
return best_acc
NN instance & Hyper Parameters¶
In [29]:
current_model_name = 'VGG16'
SAVED_MODEL_PATH = '/content/savedModels/'
vgg16_path = f'{SAVED_MODEL_PATH}{current_model_name}.wow'
# Create an instance of the class
m0_VGG16 = VGG_net(in_channels=3, num_classes=N_CLASSES)
# Set calculations to GPU device
m0_VGG16.cuda() # .to(device) ???? TODO : CHECK THE DIFFERENCE...
#_ ALTERNATIVE _
# m1_MobileNetV3 = m1_MobileNetV3.to(device)
# Loss
# criterion = FL() # TODO : CHECK THE DIFFERENCE...
#_ ALTERNATIVE _
criterion = nn.CrossEntropyLoss()
# Optimizer:
optimizer = torch.optim.Adam(m0_VGG16.parameters(), lr=LEARNING_RATE)
# Scheduler - decrease the learning rate when approaching fine-tuning stage.
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer)
# Check if we haveSAVED_MODEL_PATH already saved model
if Path(vgg16_path).exists():
epoch, loss = load_model(vgg16_path, m0_VGG16, optimizer)
else:
epoch = 0
# Check out the summary while the model is training :-)
print(epoch)
summary(m0_VGG16, SUMMARY_TENSOR_SIZE)
In [30]:
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(m0_VGG16, N_EPOCHS - epoch, train_loader, valid_loader, criterion, optimizer, scheduler, current_model_name,
writer_m0_VGG16, valid_writer_m0_VGG16, best_acc)
Out[30]:
Demonstrate Results¶
Successfull Examples¶
In [31]:
# 1=1 # STOPP
Bad Examples¶
In [32]:
import torchvision.models as models
m1_vgg11 = models.vgg11(pretrained=True)
m2_resnet101 = models.resnet101(pretrained=True)
m3_squeeze_net = models.squeezenet1_1(pretrained=True)
m4_mobilenet_v3_L = models.mobilenet_v3_large(pretrained=True)
m5_mobilenet_v3_S = models.mobilenet_v3_small(pretrained=True)
N_OF_LAST_LAYERS_TO_TRAIN = 20
In [33]:
class identity(nn.Module):
def __init__(self):
super(Identity, self).__init__()
def forward(self, x):
return x
def freeze_models_n_first_layers(model, nof_layers_to_freeze):
"""
if we want to train only the 2 last layers call function like:
freeze_models_n_first_layers(model, count_model_layers(model.parameters) - 2)
"""
freezed_counter = 0
for name, param in model.named_parameters():
if freezed_counter >= nof_layers_to_freeze:
param.requires_grad = False
else:
# Printing the layers that will be trained
# print(name, param)
# finishing func & loop, stopped freeze
return
# for param, limiter in zip(model.parameters(),range(0,nof_layers_to_freeze)):
# param.requires_grad = False
def count_model_layers(model):
counter = 0
for param in model.parameters():
counter += 1
print(counter)
return counter
freeze_models_n_first_layers(m1_vgg11, count_model_layers(m1_vgg11) - N_OF_LAST_LAYERS_TO_TRAIN)
print(m1_vgg11)
TensorBoard Writers¶
In [34]:
writer_m1_vgg11 = SummaryWriter(f'runs/m1_vgg11_{experiment}')
writer_m2_resnet101 = SummaryWriter(f'runs/m2_resnet101 _{experiment}')
writer_m3_squeeze_net = SummaryWriter(f'runs/m3_Squeeze_net_{experiment}')
writer_m4_mobilenet_v3_L = SummaryWriter(f'runs/m4_mobilenet_v3_L{experiment}')
writer_m5_mobilenet_v3_S = SummaryWriter(f'runs/m5_mobilenet_v3_S_{experiment}')
valid_writer_m1_vgg11 = SummaryWriter(f'runs/m1_vgg11_{experiment}_valid')
valid_writer_m2_resnet101 = SummaryWriter(f'runs/m2_resnet101 _{experiment}_valid')
valid_writer_m3_squeeze_net = SummaryWriter(f'runs/m3_Squeeze_net_{experiment}_valid')
valid_writer_m4_mobilenet_v3_L = SummaryWriter(f'runs/m4_mobilenet_v3_L{experiment}_valid')
valid_writer_m5_mobilenet_v3_S = SummaryWriter(f'runs/m5_mobilenet_v3_S_{experiment}_valid')
#Ensamble:
writer_model_ensembles = SummaryWriter(f'runs/model_ensembles{experiment}')
valid_writer_model_ensembles = SummaryWriter(f'runs/model_ensembles_{experiment}_valid')
VGG11¶
In [35]:
freeze_models_n_first_layers(m2_resnet101, count_model_layers(m2_resnet101) - N_OF_LAST_LAYERS_TO_TRAIN)
In [36]:
optimizer = torch.optim.Adam(m1_vgg11.parameters(), lr=LEARNING_RATE)
criterion = FL()
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
print(device)
m1_vgg11 = m1_vgg11.to(device)
if Path(f"{SAVED_MODEL_PATH}m1_vgg11.wow").exists():
epoch, loss = load_model(f"{SAVED_MODEL_PATH}m1_vgg11.wow", m1_vgg11, optimizer)
else:
epoch = 0
# Print model's architecture
summary(m1_vgg11, SUMMARY_TENSOR_SIZE)
In [37]:
# Set minimum accuracy for saving the model:
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(m1_vgg11, (N_EPOCHS - epoch), train_loader, valid_loader, criterion, optimizer, scheduler, 'm1_vgg11',
writer_m1_vgg11, valid_writer_m1_vgg11, best_acc)
Out[37]:
ResNet101¶
In [38]:
freeze_models_n_first_layers(m2_resnet101, count_model_layers(m2_resnet101) - N_OF_LAST_LAYERS_TO_TRAIN)
In [39]:
optimizer = torch.optim.Adam(m2_resnet101.parameters(), lr=LEARNING_RATE)
criterion = FL()
m2_resnet101 = m2_resnet101.to(device)
# Get model from file
if Path(f"{SAVED_MODEL_PATH}m2_resnet101.wow").exists():
epoch, loss = load_model(f"{SAVED_MODEL_PATH}m2_resnet101.wow", m2_resnet101, optimizer)
else:
epoch = 0
# Print model's architecture
summary(m2_resnet101, SUMMARY_TENSOR_SIZE)
In [40]:
# Set minimum accuracy for saving the model:
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(m2_resnet101, N_EPOCHS - epoch, train_loader, valid_loader, criterion, optimizer, scheduler, 'm2_resnet101',
writer_m2_resnet101, valid_writer_m2_resnet101, best_acc)
Out[40]:
SqueezeNet 1.1¶
In [41]:
freeze_models_n_first_layers(m3_squeeze_net, count_model_layers(m3_squeeze_net) - N_OF_LAST_LAYERS_TO_TRAIN)
In [42]:
optimizer = torch.optim.Adam(m3_squeeze_net.parameters(), lr=LEARNING_RATE)
criterion = FL()
m3_squeeze_net = m3_squeeze_net.to(device)
# Get model from file
if Path(f"{SAVED_MODEL_PATH}m3_squeeze_net.wow").exists():
epoch, loss = load_model(f"{SAVED_MODEL_PATH}m3_squeeze_net.wow", m3_squeeze_net, optimizer)
else:
epoch = 0
# Print model's architecture
summary(m3_squeeze_net, SUMMARY_TENSOR_SIZE)
In [43]:
# Set minimum accuracy for saving the model:
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(m3_squeeze_net, N_EPOCHS - epoch, train_loader, valid_loader, criterion, optimizer, scheduler, 'm3_squeeze_net',
writer_m3_squeeze_net, valid_writer_m3_squeeze_net, best_acc)
Out[43]:
mobilenet_v3_Large¶
In [44]:
freeze_models_n_first_layers(m4_mobilenet_v3_L, count_model_layers(m4_mobilenet_v3_L) - N_OF_LAST_LAYERS_TO_TRAIN)
In [45]:
optimizer = torch.optim.Adam(m4_mobilenet_v3_L.parameters(), lr=LEARNING_RATE)
criterion = FL()
m4_mobilenet_v3_L = m4_mobilenet_v3_L.to(device)
# Get model from file
if Path(f"{SAVED_MODEL_PATH}m4_mobilenet_v3_L.wow").exists():
epoch, loss = load_model(f"{SAVED_MODEL_PATH}m4_mobilenet_v3_L.wow", m4_mobilenet_v3_L, optimizer)
else:
epoch = 0
# Print model's architecture
summary(m4_mobilenet_v3_L, SUMMARY_TENSOR_SIZE)
In [46]:
# Set minimum accuracy for saving the model:
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(m4_mobilenet_v3_L, N_EPOCHS - epoch, train_loader, valid_loader, criterion, optimizer, scheduler, 'm4_mobilenet_v3_L',
writer_m4_mobilenet_v3_L, valid_writer_m4_mobilenet_v3_L, best_acc)
Out[46]:
mobilenet_v3_Small¶
In [47]:
freeze_models_n_first_layers(m5_mobilenet_v3_S, count_model_layers(m5_mobilenet_v3_S) - N_OF_LAST_LAYERS_TO_TRAIN)
In [48]:
optimizer = torch.optim.Adam(m5_mobilenet_v3_S.parameters(), lr=LEARNING_RATE)
criterion = FL()
m5_mobilenet_v3_S = m5_mobilenet_v3_S.to(device)
# Get model from file
if Path(f"{SAVED_MODEL_PATH}m5_mobilenet_v3_S.wow").exists():
epoch, loss = load_model(f"{SAVED_MODEL_PATH}m5_mobilenet_v3_S.wow", m5_mobilenet_v3_S, optimizer)
else:
epoch = 0
# Print model's architecture
summary(m5_mobilenet_v3_S, SUMMARY_TENSOR_SIZE)
In [49]:
# Set minimum accuracy for saving the model:
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(m5_mobilenet_v3_S, N_EPOCHS - epoch, train_loader, valid_loader, criterion, optimizer, scheduler, 'm5_mobilenet_v3_S',
writer_m5_mobilenet_v3_S, valid_writer_m5_mobilenet_v3_S, best_acc)
Out[49]:
Model Ensembles¶
Ensemble Class¶
In [50]:
from torch.nn import functional as F
class MyEnsemble(nn.Module):
def __init__(self, model_1, model_2, model_3, model_4, model_5, nb_classes):
super(MyEnsemble,self).__init__()
self.model_1 = model_1
self.model_2 = model_2
self.model_3 = model_3
self.model_4 = model_4
self.model_5 = model_5
#Now Remove the Last layer
self.model_1 = nn.Identity()
self.model_2 = nn.Identity()
self.model_3 = nn.Identity()
self.model_4 = nn.Identity()
self.model_5 = nn.Identity()
# choosed from the error
sum_of_all_models_input_sizes = 752640 # ???? Right???
self.classifier = nn.Linear(sum_of_all_models_input_sizes ,nb_classes)
def forward(self, x):
x.clone() # clone to make sure x is not changed by inplace methods
x1 = self.model_1(x)
x1 = x1.view(x1.size(0), -1)
x2 = self.model_2(x)
x2 = x2.view(x2.size(0), -1)
x3 = self.model_3(x)
x3 = x3.view(x3.size(0), -1)
x4 = self.model_4(x)
x4 = x4.view(x4.size(0), -1)
x5 = self.model_5(x)
x5 = x5.view(x5.size(0), -1)
#final
x = torch.cat((x1, x2, x3, x4, x5), dim=1)
x = self.classifier(F.relu(x))
return x
model_ensembles = MyEnsemble(m1_vgg11,m2_resnet101, m3_squeeze_net, m4_mobilenet_v3_L, m5_mobilenet_v3_S, N_CLASSES)
In [51]:
# Fetch batch
data, targets = next(iter(train_loader))
model_ensembles(data.float())
Out[51]:
Ensemble Training¶
In [52]:
optimizer = torch.optim.Adam(model_ensembles.parameters(), lr=LEARNING_RATE)
criterion = FL()
model_ensembles = model_ensembles.to(device)
# Get model from file
if Path(f"{SAVED_MODEL_PATH}model_ensembles.wow").exists():
epoch, loss = load_model(f"{SAVED_MODEL_PATH}model_ensembles.wow", model_ensembles, optimizer)
else:
epoch = 0
summary(model_ensembles, SUMMARY_TENSOR_SIZE)
best_acc = MINIMUM_ACC_4RECORDING
train_n_epochs(model_ensembles, N_EPOCHS - epoch, train_loader, valid_loader, criterion, optimizer, scheduler, 'model_ensembles',
writer_model_ensembles, valid_writer_model_ensembles, best_acc)
Out[52]:
Summary¶
In this assigment we've seen how to:
- Build our own costum pytorch neural network
- Save pytorch's model and load from disk
- Use Tensorboard for monitoring training progress in real time
- Download pretrained model, and use them
- Build an ensemble of models, which usually is better than those who build it.
- When models don't train - try to overfit a small batch
- How to fine-tune & Transfer learning a model
- Manage GPU memory, by del batches of used info + Clear it's cache
- Revert back to best model once acc is not rising after N rounds
Screen Shots¶
23May2021
TensorBoard¶
SNIPPETS & References¶
In [53]:
# TRAIN MODEL LOOP
# DEBUG = False
# # Train the model
# for epoch in range(N_EPOCHS):
# avg_loss = 0
# cnt = 0
# for images, targets in train_loader:
# images = images.cuda()
# targets = targets.type(torch.LongTensor)
# targets = targets.cuda()
# # Forward
# optimizer.zero_grad()
# outputs = vgg16(images)
# # Prints:
# if DEBUG:
# print('output',output.detach().cpu().numpy(),
# 'shape', output.detach().cpu().numpy().shape )
# print('target',target.detach().cpu().numpy(),
# 'shape', target.detach().cpu().numpy().shape )
# # Loss
# loss = cost(outputs, targets)
# avg_loss += loss.data
# cnt += 1
# print("[E: %d] loss: %f, avg_loss: %f" % (epoch, loss.data, avg_loss/cnt))
# # Backward
# loss.backward()
# # Optimize
# optimizer.step()
# scheduler.step(avg_loss)
# # for images, targets in train_loader:
# # for i in range(batch_size):
# # plt.imshow(images[i])
# # plt.title(targets[i])
# # plt.show()
# # https://stackoverflow.com/questions/55762581/expected-object-of-scalar-type-long-but-got-scalar-type-byte-for-argument-2-ta
In [54]:
# # Test The Model
# # Set model to evaluation mode
# vgg16.eval()
# # counters:
# correct = 0
# total = 0
# for images, labels in valid_loader:
# images = images.cuda()
# outputs = vgg16(images)
# _, predicted = torch.max(outputs.data, 1)
# total += labels.size(0)
# correct += (predicted.cpu() == labels).sum()
# print(predicted, labels, correct, total)
# print("avg acc: %f" % (100* correct/total))